<!DOCTYPE html>



  


<html class="theme-next pisces use-motion" lang="zh-Hans">
<head>
  <!-- hexo-inject:begin --><!-- hexo-inject:end --><meta charset="UTF-8"/>
<meta http-equiv="X-UA-Compatible" content="IE=edge" />
<meta name="viewport" content="width=device-width, initial-scale=1, maximum-scale=1"/>
<meta name="theme-color" content="#222">









<meta http-equiv="Cache-Control" content="no-transform" />
<meta http-equiv="Cache-Control" content="no-siteapp" />
















  
  
  <link href="/lib/fancybox/source/jquery.fancybox.css?v=2.1.5" rel="stylesheet" type="text/css" />







<link href="/lib/font-awesome/css/font-awesome.min.css?v=4.6.2" rel="stylesheet" type="text/css" />

<link href="/css/main.css?v=5.1.3" rel="stylesheet" type="text/css" />


  <link rel="apple-touch-icon" sizes="180x180" href="/images/avatar.jpg?v=5.1.3">


  <link rel="icon" type="image/png" sizes="32x32" href="/images/avatar.jpg?v=5.1.3">


  <link rel="icon" type="image/png" sizes="16x16" href="/images/avatar.jpg?v=5.1.3">


  <link rel="mask-icon" href="/images/avatar.jpg?v=5.1.3" color="#222">





  <meta name="keywords" content="Hadoop," />










<meta name="description" content="环境 操作系统：Ubuntu 14.04 Hadoop版本：Hadoop 2.6.5 JDK版本：OpenJDK 1.7  创建Hadoop用户创建用户1sudo useradd -m hadoop 为新用户设置密码1sudo passwd hadoop 为新用户添加权限1sudo adduser hadoop sudo  如果失败，则切换到root用户下，修改/etc/sudoers文件，将ha">
<meta name="keywords" content="Hadoop">
<meta property="og:type" content="article">
<meta property="og:title" content="Hadoop 安装和使用">
<meta property="og:url" content="http://fighterhit.github.io/2017/11/12/other_notes/Hadoop-安装和使用/index.html">
<meta property="og:site_name" content="Fighter&#39;s Blog">
<meta property="og:description" content="环境 操作系统：Ubuntu 14.04 Hadoop版本：Hadoop 2.6.5 JDK版本：OpenJDK 1.7  创建Hadoop用户创建用户1sudo useradd -m hadoop 为新用户设置密码1sudo passwd hadoop 为新用户添加权限1sudo adduser hadoop sudo  如果失败，则切换到root用户下，修改/etc/sudoers文件，将ha">
<meta property="og:locale" content="zh-Hans">
<meta property="og:image" content="http://upload-images.jianshu.io/upload_images/137261-14d8fa9cf9e8fec4.jpg?imageMogr2/auto-orient/strip%7CimageView2/2/w/1240">
<meta property="og:image" content="http://upload-images.jianshu.io/upload_images/137261-21dda5b3a1dcbbd1.jpg?imageMogr2/auto-orient/strip%7CimageView2/2/w/1240">
<meta property="og:image" content="http://upload-images.jianshu.io/upload_images/137261-0ea47d412d606a21.jpg?imageMogr2/auto-orient/strip%7CimageView2/2/w/1240">
<meta property="og:image" content="http://upload-images.jianshu.io/upload_images/137261-2e2d2f39dc26cf88.jpg?imageMogr2/auto-orient/strip%7CimageView2/2/w/1240">
<meta property="og:image" content="http://upload-images.jianshu.io/upload_images/137261-40144e2790b61b17.jpg?imageMogr2/auto-orient/strip%7CimageView2/2/w/1240">
<meta property="og:image" content="http://upload-images.jianshu.io/upload_images/137261-5799532edfea6048.jpg?imageMogr2/auto-orient/strip%7CimageView2/2/w/1240">
<meta property="og:image" content="http://upload-images.jianshu.io/upload_images/137261-27ba96b62b566db8.jpg?imageMogr2/auto-orient/strip%7CimageView2/2/w/1240">
<meta property="og:image" content="http://upload-images.jianshu.io/upload_images/137261-27a23c6c960a1b9c.jpg?imageMogr2/auto-orient/strip%7CimageView2/2/w/1240">
<meta property="og:image" content="http://upload-images.jianshu.io/upload_images/137261-63927a3add57b2a3.jpg?imageMogr2/auto-orient/strip%7CimageView2/2/w/1240">
<meta property="og:image" content="http://upload-images.jianshu.io/upload_images/137261-bdf67932c07a1efa.jpg?imageMogr2/auto-orient/strip%7CimageView2/2/w/1240">
<meta property="og:image" content="http://upload-images.jianshu.io/upload_images/137261-d3bb6626500d065c.jpg?imageMogr2/auto-orient/strip%7CimageView2/2/w/1240">
<meta property="og:image" content="http://upload-images.jianshu.io/upload_images/137261-3863016cb756abe8.jpg?imageMogr2/auto-orient/strip%7CimageView2/2/w/1240">
<meta property="og:image" content="http://upload-images.jianshu.io/upload_images/137261-912a1d7285b229a4.jpg?imageMogr2/auto-orient/strip%7CimageView2/2/w/1240">
<meta property="og:image" content="http://upload-images.jianshu.io/upload_images/137261-55c3161dffe70bb3.jpg?imageMogr2/auto-orient/strip%7CimageView2/2/w/1240">
<meta property="og:image" content="http://upload-images.jianshu.io/upload_images/137261-bf6fc9e6ee002389.jpg?imageMogr2/auto-orient/strip%7CimageView2/2/w/1240">
<meta property="og:image" content="http://upload-images.jianshu.io/upload_images/137261-31e7db693f954734.jpg?imageMogr2/auto-orient/strip%7CimageView2/2/w/1240">
<meta property="og:image" content="http://upload-images.jianshu.io/upload_images/137261-342e53f177650700.jpg?imageMogr2/auto-orient/strip%7CimageView2/2/w/1240">
<meta property="og:updated_time" content="2018-06-28T12:05:20.495Z">
<meta name="twitter:card" content="summary">
<meta name="twitter:title" content="Hadoop 安装和使用">
<meta name="twitter:description" content="环境 操作系统：Ubuntu 14.04 Hadoop版本：Hadoop 2.6.5 JDK版本：OpenJDK 1.7  创建Hadoop用户创建用户1sudo useradd -m hadoop 为新用户设置密码1sudo passwd hadoop 为新用户添加权限1sudo adduser hadoop sudo  如果失败，则切换到root用户下，修改/etc/sudoers文件，将ha">
<meta name="twitter:image" content="http://upload-images.jianshu.io/upload_images/137261-14d8fa9cf9e8fec4.jpg?imageMogr2/auto-orient/strip%7CimageView2/2/w/1240">



<script type="text/javascript" id="hexo.configurations">
  var NexT = window.NexT || {};
  var CONFIG = {
    root: '/',
    scheme: 'Pisces',
    version: '5.1.3',
    sidebar: {"position":"left","display":"post","offset":12,"b2t":false,"scrollpercent":false,"onmobile":false},
    fancybox: true,
    tabs: true,
    motion: {"enable":true,"async":false,"transition":{"post_block":"fadeIn","post_header":"slideDownIn","post_body":"slideDownIn","coll_header":"slideLeftIn","sidebar":"slideUpIn"}},
    duoshuo: {
      userId: '0',
      author: '博主'
    },
    algolia: {
      applicationID: '',
      apiKey: '',
      indexName: '',
      hits: {"per_page":10},
      labels: {"input_placeholder":"Search for Posts","hits_empty":"We didn't find any results for the search: ${query}","hits_stats":"${hits} results found in ${time} ms"}
    }
  };
</script>



  <link rel="canonical" href="http://fighterhit.github.io/2017/11/12/other_notes/Hadoop-安装和使用/"/>





  <title>Hadoop 安装和使用 | Fighter's Blog</title><!-- hexo-inject:begin --><!-- hexo-inject:end -->
  








</head>

<body itemscope itemtype="http://schema.org/WebPage" lang="zh-Hans">

  
  
    
  

  <!-- hexo-inject:begin --><!-- hexo-inject:end --><div class="container sidebar-position-left page-post-detail">
    <div class="headband"></div>

    <header id="header" class="header" itemscope itemtype="http://schema.org/WPHeader">
      <div class="header-inner"><div class="site-brand-wrapper">
  <div class="site-meta ">
    

    <div class="custom-logo-site-title">
      <a href="/"  class="brand" rel="start">
        <span class="logo-line-before"><i></i></span>
        <span class="site-title">Fighter's Blog</span>
        <span class="logo-line-after"><i></i></span>
      </a>
    </div>
      
        <p class="site-subtitle">深度沉迷学习</p>
      
  </div>

  <div class="site-nav-toggle">
    <button>
      <span class="btn-bar"></span>
      <span class="btn-bar"></span>
      <span class="btn-bar"></span>
    </button>
  </div>
</div>

<nav class="site-nav">
  

  
    <ul id="menu" class="menu">
      
        
        <li class="menu-item menu-item-home">
          <a href="/" rel="section">
            
              <i class="menu-item-icon fa fa-fw fa-home"></i> <br />
            
            首页
          </a>
        </li>
      
        
        <li class="menu-item menu-item-about">
          <a href="/about/" rel="section">
            
              <i class="menu-item-icon fa fa-fw fa-user"></i> <br />
            
            关于
          </a>
        </li>
      
        
        <li class="menu-item menu-item-tags">
          <a href="/tags/" rel="section">
            
              <i class="menu-item-icon fa fa-fw fa-tags"></i> <br />
            
            标签
          </a>
        </li>
      
        
        <li class="menu-item menu-item-categories">
          <a href="/categories/" rel="section">
            
              <i class="menu-item-icon fa fa-fw fa-th"></i> <br />
            
            分类
          </a>
        </li>
      
        
        <li class="menu-item menu-item-archives">
          <a href="/archives/" rel="section">
            
              <i class="menu-item-icon fa fa-fw fa-archive"></i> <br />
            
            归档
          </a>
        </li>
      

      
    </ul>
  

  
</nav>



 </div>
    </header>

    <main id="main" class="main">
      <div class="main-inner">
        <div class="content-wrap">
          <div id="content" class="content">
            

  <div id="posts" class="posts-expand">
    

  

  
  
  

  <article class="post post-type-normal" itemscope itemtype="http://schema.org/Article">
  
  
  
  <div class="post-block">
    <link itemprop="mainEntityOfPage" href="http://fighterhit.github.io/2017/11/12/other_notes/Hadoop-安装和使用/">

    <span hidden itemprop="author" itemscope itemtype="http://schema.org/Person">
      <meta itemprop="name" content="Fighter.">
      <meta itemprop="description" content="">
      <meta itemprop="image" content="/images/avatar.jpg">
    </span>

    <span hidden itemprop="publisher" itemscope itemtype="http://schema.org/Organization">
      <meta itemprop="name" content="Fighter's Blog">
    </span>

    
      <header class="post-header">

        
        
          <h1 class="post-title" itemprop="name headline">Hadoop 安装和使用</h1>
        

        <div class="post-meta">
          <span class="post-time">
            
              <span class="post-meta-item-icon">
                <i class="fa fa-calendar-o"></i>
              </span>
              
                <span class="post-meta-item-text">发表于</span>
              
              <time title="创建于" itemprop="dateCreated datePublished" datetime="2017-11-12T12:07:00+08:00">
                2017-11-12
              </time>
            

            

            
          </span>

          
            <span class="post-category" >
            
              <span class="post-meta-divider">|</span>
            
              <span class="post-meta-item-icon">
                <i class="fa fa-folder-o"></i>
              </span>
              
                <span class="post-meta-item-text">分类于</span>
              
              
                <span itemprop="about" itemscope itemtype="http://schema.org/Thing">
                  <a href="/categories/其它笔记/" itemprop="url" rel="index">
                    <span itemprop="name">其它笔记</span>
                  </a>
                </span>

                
                
              
            </span>
          

          
            
          

          
          

          
            <span class="post-meta-divider">|</span>
            <span class="page-pv"><i class="fa fa-file-o"></i>
            <span class="busuanzi-value" id="busuanzi_value_page_pv" ></span>
            </span>
          

          

          

        </div>
      </header>
    

    
    
    
    <div class="post-body" itemprop="articleBody">

      
      

      
        <h2 id="环境"><a href="#环境" class="headerlink" title="环境"></a>环境</h2><ul>
<li>操作系统：Ubuntu 14.04</li>
<li>Hadoop版本：Hadoop 2.6.5</li>
<li>JDK版本：OpenJDK 1.7</li>
</ul>
<h2 id="创建Hadoop用户"><a href="#创建Hadoop用户" class="headerlink" title="创建Hadoop用户"></a>创建Hadoop用户</h2><h4 id="创建用户"><a href="#创建用户" class="headerlink" title="创建用户"></a>创建用户</h4><figure class="highlight plain"><table><tr><td class="gutter"><pre><div class="line">1</div></pre></td><td class="code"><pre><div class="line">sudo useradd -m hadoop</div></pre></td></tr></table></figure>
<h4 id="为新用户设置密码"><a href="#为新用户设置密码" class="headerlink" title="为新用户设置密码"></a>为新用户设置密码</h4><figure class="highlight plain"><table><tr><td class="gutter"><pre><div class="line">1</div></pre></td><td class="code"><pre><div class="line">sudo passwd hadoop</div></pre></td></tr></table></figure>
<h4 id="为新用户添加权限"><a href="#为新用户添加权限" class="headerlink" title="为新用户添加权限"></a>为新用户添加权限</h4><figure class="highlight plain"><table><tr><td class="gutter"><pre><div class="line">1</div></pre></td><td class="code"><pre><div class="line">sudo adduser hadoop sudo</div></pre></td></tr></table></figure>
<blockquote>
<p>如果失败，则切换到root用户下，修改/etc/sudoers文件，将hadoop用户添加进去。</p>
</blockquote>
<a id="more"></a>
<h2 id="安装SSH-Server、配置SSH免密钥登录"><a href="#安装SSH-Server、配置SSH免密钥登录" class="headerlink" title="安装SSH Server、配置SSH免密钥登录"></a>安装SSH Server、配置SSH免密钥登录</h2><p>Ubuntu默认安装了SSH client，还需要安装SSH server。集群、单节点模式都需要用到SSH无密码登陆。</p>
<figure class="highlight plain"><table><tr><td class="gutter"><pre><div class="line">1</div></pre></td><td class="code"><pre><div class="line">sudo apt-get install openssh-server</div></pre></td></tr></table></figure>
<h4 id="设置登录到本机"><a href="#设置登录到本机" class="headerlink" title="设置登录到本机"></a>设置登录到本机</h4><p>运行命令：</p>
<figure class="highlight plain"><table><tr><td class="gutter"><pre><div class="line">1</div></pre></td><td class="code"><pre><div class="line">ssh localhost</div></pre></td></tr></table></figure>
<p><img src="http://upload-images.jianshu.io/upload_images/137261-14d8fa9cf9e8fec4.jpg?imageMogr2/auto-orient/strip%7CimageView2/2/w/1240" alt="设置登录到本机"></p>
<p>会有如下提示，输入yes。</p>
<p><img src="http://upload-images.jianshu.io/upload_images/137261-21dda5b3a1dcbbd1.jpg?imageMogr2/auto-orient/strip%7CimageView2/2/w/1240" alt="设置登录到本机"></p>
<p>SSH首次登陆提示。然后按提示输入密码hadoop，这样就登陆到本机了。但这样的登陆是需要密码的，需要配置成无密码登陆。</p>
<h4 id="配置免密钥登录"><a href="#配置免密钥登录" class="headerlink" title="配置免密钥登录"></a>配置免密钥登录</h4><p>运行命令：</p>
<figure class="highlight plain"><table><tr><td class="gutter"><pre><div class="line">1</div><div class="line">2</div></pre></td><td class="code"><pre><div class="line">cd ~/.ssh</div><div class="line">ssh-keyen</div></pre></td></tr></table></figure>
<p>连按三次回车。</p>
<p><img src="http://upload-images.jianshu.io/upload_images/137261-0ea47d412d606a21.jpg?imageMogr2/auto-orient/strip%7CimageView2/2/w/1240" alt="生成密钥"></p>
<p>再运行命令（将公钥拷贝到本地）：</p>
<figure class="highlight plain"><table><tr><td class="gutter"><pre><div class="line">1</div></pre></td><td class="code"><pre><div class="line">ssh-copy-id -i ~/.ssh/id_rsa.pub localhost</div></pre></td></tr></table></figure>
<p><img src="http://upload-images.jianshu.io/upload_images/137261-2e2d2f39dc26cf88.jpg?imageMogr2/auto-orient/strip%7CimageView2/2/w/1240" alt="将公钥拷贝到本地"></p>
<p>此时再执行 ssh localhost 命令，就可以直接登陆了，如下图所示。</p>
<p><img src="http://upload-images.jianshu.io/upload_images/137261-40144e2790b61b17.jpg?imageMogr2/auto-orient/strip%7CimageView2/2/w/1240" alt="免密钥登录本地"></p>
<h2 id="配置Java环境"><a href="#配置Java环境" class="headerlink" title="配置Java环境"></a>配置Java环境</h2><h4 id="下载JDK"><a href="#下载JDK" class="headerlink" title="下载JDK"></a>下载JDK</h4><p>根据 <a href="https://wiki.apache.org/hadoop/HadoopJavaVersions" target="_blank" rel="noopener">Hadoop Java Versions</a> 中的罗列的 Tested JDK 项可知，Hadoop 支持 openjdk 1.7，因此下载 openjdk 1.7。运行如下命令，下载 openjdk 1.7：</p>
<figure class="highlight plain"><table><tr><td class="gutter"><pre><div class="line">1</div></pre></td><td class="code"><pre><div class="line">sudo apt-get install openjdk-7-jre openjdk-7-jdk</div></pre></td></tr></table></figure>
<p>JDK默认安装在 /usr/lib/jvm/java-7-openjdk-amd64 ，运行 java -version 出现下图，表示 jdk 安装成功。</p>
<p><img src="http://upload-images.jianshu.io/upload_images/137261-5799532edfea6048.jpg?imageMogr2/auto-orient/strip%7CimageView2/2/w/1240" alt="jdk安装成功"></p>
<h4 id="配置JDK环境变量"><a href="#配置JDK环境变量" class="headerlink" title="配置JDK环境变量"></a>配置JDK环境变量</h4><p>运行如下命令，来编辑全局环境变量文件</p>
<figure class="highlight plain"><table><tr><td class="gutter"><pre><div class="line">1</div></pre></td><td class="code"><pre><div class="line">sudo vim /etc/environment</div></pre></td></tr></table></figure>
<p>在文件末尾添加一行：</p>
<figure class="highlight plain"><table><tr><td class="gutter"><pre><div class="line">1</div></pre></td><td class="code"><pre><div class="line">JAVA_HOME=&quot;/usr/lib/jvm/java-7-openjdk-amd64&quot;</div></pre></td></tr></table></figure>
<p>注销或重启后，在终端执行 echo $JAVA_HOME 即可查看环境变量是否生效，生效后的效果如下图。</p>
<p><img src="http://upload-images.jianshu.io/upload_images/137261-27ba96b62b566db8.jpg?imageMogr2/auto-orient/strip%7CimageView2/2/w/1240" alt="配置JAVA_HOME"></p>
<h2 id="安装、配置-Hadoop-2-6-5"><a href="#安装、配置-Hadoop-2-6-5" class="headerlink" title="安装、配置 Hadoop 2.6.5"></a>安装、配置 Hadoop 2.6.5</h2><h4 id="下载-Hadoop"><a href="#下载-Hadoop" class="headerlink" title="下载 Hadoop"></a>下载 Hadoop</h4><p>从 <a href="http://hadoop.apache.org/releases.html" target="_blank" rel="noopener">Hadoop官网</a> 上下载 Hadoop二进制压缩包 hadoop-2.6.5.tar.gz 到本地目录中。下载完成后解压缩：</p>
<figure class="highlight plain"><table><tr><td class="gutter"><pre><div class="line">1</div><div class="line">2</div><div class="line">3</div><div class="line">4</div><div class="line">5</div></pre></td><td class="code"><pre><div class="line">cd ~/software/</div><div class="line">sudo tar -zxvf hadoop-2.6.5.tar.gz -C /usr/local   # 解压到/usr/local中</div><div class="line">cd /usr/local</div><div class="line">sudo mv hadoop-2.6.5 hadoop      # 将文件名改为hadoop</div><div class="line">sudo chown -R hadoop:hadoop /usr/local/hadoop       # 修改文件权限</div></pre></td></tr></table></figure>
<p>Hadoop解压后即可使用，输入如下命令Hadoop检查是否可用：</p>
<figure class="highlight plain"><table><tr><td class="gutter"><pre><div class="line">1</div></pre></td><td class="code"><pre><div class="line">/usr/local/hadoop/bin/hadoop</div></pre></td></tr></table></figure>
<p>可用则会显示命令行的用法：</p>
<p><img src="http://upload-images.jianshu.io/upload_images/137261-27a23c6c960a1b9c.jpg?imageMogr2/auto-orient/strip%7CimageView2/2/w/1240" alt="Hadoop安装成功"></p>
<h4 id="Hadoop单机配置"><a href="#Hadoop单机配置" class="headerlink" title="Hadoop单机配置"></a>Hadoop单机配置</h4><p>Hadoop默认配置是以非分布式模式运行，即单Java进程，方便进行调试。可以执行附带的例子WordCount来查看Hadoop的运行。例子将Hadoop的配置文件作为输入文件，统计符合正则表达式dfs[a-z.]+的单词的出现次数。</p>
<figure class="highlight plain"><table><tr><td class="gutter"><pre><div class="line">1</div><div class="line">2</div><div class="line">3</div><div class="line">4</div><div class="line">5</div></pre></td><td class="code"><pre><div class="line">cd /usr/local/hadoop</div><div class="line">mkdir input</div><div class="line">cp etc/hadoop/*.xml input</div><div class="line">bin/hadoop jar share/hadoop/mapreduce/hadoop-mapreduce-examples-2.6.5.jar grep input output &apos;dfs[a-z.]+&apos;</div><div class="line">cat ./output/*</div></pre></td></tr></table></figure>
<p>执行成功后如下所示，输出了作业的相关信息，输出的结果是符合正则的单词 dfsadmin，出现了1次。</p>
<p><img src="http://upload-images.jianshu.io/upload_images/137261-63927a3add57b2a3.jpg?imageMogr2/auto-orient/strip%7CimageView2/2/w/1240" alt="Hadoop单机实例运行结果"></p>
<h4 id="Hadoop伪分布式配置"><a href="#Hadoop伪分布式配置" class="headerlink" title="Hadoop伪分布式配置"></a>Hadoop伪分布式配置</h4><p>Hadoop可以在单节点上以伪分布式的方式运行，此时Hadoop进程以分离的Java进程来运行，节点既是NameNode也是DataNode。这种情况下，需要修改 etc/hadoop/ 目录下core-site.xml和hdfs-site.xml。Hadoop的配置文件是xml格式，声明property的name和value。</p>
<h5 id="修改配置文件-etc-hadoop-core-site-xml"><a href="#修改配置文件-etc-hadoop-core-site-xml" class="headerlink" title="修改配置文件 etc/hadoop/core-site.xml"></a>修改配置文件 etc/hadoop/core-site.xml</h5><p>将 etc/hadoop/core-site.xml 文件中的 <configuration></configuration> 修改为：</p>
<figure class="highlight xml"><table><tr><td class="gutter"><pre><div class="line">1</div><div class="line">2</div><div class="line">3</div><div class="line">4</div><div class="line">5</div><div class="line">6</div><div class="line">7</div><div class="line">8</div><div class="line">9</div><div class="line">10</div><div class="line">11</div></pre></td><td class="code"><pre><div class="line"><span class="tag">&lt;<span class="name">configuration</span>&gt;</span></div><div class="line">    <span class="tag">&lt;<span class="name">property</span>&gt;</span></div><div class="line">        <span class="tag">&lt;<span class="name">name</span>&gt;</span>hadoop.tmp.dir<span class="tag">&lt;/<span class="name">name</span>&gt;</span></div><div class="line">        <span class="tag">&lt;<span class="name">value</span>&gt;</span>file:/usr/local/hadoop/tmp<span class="tag">&lt;/<span class="name">value</span>&gt;</span></div><div class="line">        <span class="tag">&lt;<span class="name">description</span>&gt;</span>some description<span class="tag">&lt;/<span class="name">description</span>&gt;</span></div><div class="line">    <span class="tag">&lt;/<span class="name">property</span>&gt;</span></div><div class="line">    <span class="tag">&lt;<span class="name">property</span>&gt;</span></div><div class="line">        <span class="tag">&lt;<span class="name">name</span>&gt;</span>fs.defaultFS<span class="tag">&lt;/<span class="name">name</span>&gt;</span></div><div class="line">        <span class="tag">&lt;<span class="name">value</span>&gt;</span>hdfs://localhost:9000<span class="tag">&lt;/<span class="name">value</span>&gt;</span></div><div class="line">    <span class="tag">&lt;/<span class="name">property</span>&gt;</span></div><div class="line"><span class="tag">&lt;/<span class="name">configuration</span>&gt;</span></div></pre></td></tr></table></figure>
<p><img src="http://upload-images.jianshu.io/upload_images/137261-bdf67932c07a1efa.jpg?imageMogr2/auto-orient/strip%7CimageView2/2/w/1240" alt="core-site.xml配置"></p>
<h5 id="修改配置文件etc-hadoop-hdfs-site-xml"><a href="#修改配置文件etc-hadoop-hdfs-site-xml" class="headerlink" title="修改配置文件etc/hadoop/hdfs-site.xml"></a>修改配置文件etc/hadoop/hdfs-site.xml</h5><p>将 etc/hadoop/hdfs-site.xml 文件中的 <configuration></configuration> 修改为：</p>
<figure class="highlight xml"><table><tr><td class="gutter"><pre><div class="line">1</div><div class="line">2</div><div class="line">3</div><div class="line">4</div><div class="line">5</div><div class="line">6</div><div class="line">7</div><div class="line">8</div><div class="line">9</div><div class="line">10</div><div class="line">11</div><div class="line">12</div><div class="line">13</div><div class="line">14</div><div class="line">15</div><div class="line">16</div><div class="line">17</div></pre></td><td class="code"><pre><div class="line"></div><div class="line"><span class="tag">&lt;<span class="name">configuration</span>&gt;</span></div><div class="line">    <span class="tag">&lt;<span class="name">property</span>&gt;</span></div><div class="line">        <span class="tag">&lt;<span class="name">name</span>&gt;</span>dfs.replication<span class="tag">&lt;/<span class="name">name</span>&gt;</span></div><div class="line">        <span class="tag">&lt;<span class="name">value</span>&gt;</span>1<span class="tag">&lt;/<span class="name">value</span>&gt;</span></div><div class="line">    <span class="tag">&lt;/<span class="name">property</span>&gt;</span></div><div class="line"></div><div class="line">    <span class="tag">&lt;<span class="name">property</span>&gt;</span></div><div class="line">        <span class="tag">&lt;<span class="name">name</span>&gt;</span>dfs.namenode.name.dir<span class="tag">&lt;/<span class="name">name</span>&gt;</span></div><div class="line">        <span class="tag">&lt;<span class="name">value</span>&gt;</span>file:/usr/local/hadoop/tmp/dfs/name<span class="tag">&lt;/<span class="name">value</span>&gt;</span></div><div class="line">    <span class="tag">&lt;/<span class="name">property</span>&gt;</span></div><div class="line">    </div><div class="line">    <span class="tag">&lt;<span class="name">property</span>&gt;</span></div><div class="line">        <span class="tag">&lt;<span class="name">name</span>&gt;</span>dfs.datanode.data.dir<span class="tag">&lt;/<span class="name">name</span>&gt;</span></div><div class="line">        <span class="tag">&lt;<span class="name">value</span>&gt;</span>file:/usr/local/hadoop/tmp/dfs/data<span class="tag">&lt;/<span class="name">value</span>&gt;</span></div><div class="line">    <span class="tag">&lt;/<span class="name">property</span>&gt;</span></div><div class="line"><span class="tag">&lt;/<span class="name">configuration</span>&gt;</span></div></pre></td></tr></table></figure>
<p><img src="http://upload-images.jianshu.io/upload_images/137261-d3bb6626500d065c.jpg?imageMogr2/auto-orient/strip%7CimageView2/2/w/1240" alt="hdfs-site.xml配置"></p>
<p>配置完成后，初始化Hadoop文件系统 hdfs：</p>
<figure class="highlight plain"><table><tr><td class="gutter"><pre><div class="line">1</div></pre></td><td class="code"><pre><div class="line">./bin/hdfs namenode -format</div></pre></td></tr></table></figure>
<p>运行成功效果如图：</p>
<p><img src="http://upload-images.jianshu.io/upload_images/137261-3863016cb756abe8.jpg?imageMogr2/auto-orient/strip%7CimageView2/2/w/1240" alt="hdfs格式化"></p>
<p>启动 NameNode 和 DataNode：</p>
<figure class="highlight plain"><table><tr><td class="gutter"><pre><div class="line">1</div></pre></td><td class="code"><pre><div class="line">./sbin/start-dfs.sh</div></pre></td></tr></table></figure>
<p>运行成功效果如下图，可以通过 jps 命令查看 NameNode、DataNode 和 SecondNameNode 进程信息：</p>
<p><img src="http://upload-images.jianshu.io/upload_images/137261-912a1d7285b229a4.jpg?imageMogr2/auto-orient/strip%7CimageView2/2/w/1240" alt="启动 NameNode 和 DataNode"></p>
<p>还可以通过 Web 方式查看 Hadoop 相关信息：</p>
<p><img src="http://upload-images.jianshu.io/upload_images/137261-55c3161dffe70bb3.jpg?imageMogr2/auto-orient/strip%7CimageView2/2/w/1240" alt="Web 方式查看 Hadoop 相关信息"></p>
<h5 id="Hadoop伪分布式实例-WordCount"><a href="#Hadoop伪分布式实例-WordCount" class="headerlink" title="Hadoop伪分布式实例-WordCount"></a>Hadoop伪分布式实例-WordCount</h5><p>首先创建所需的目录，并把 /etc/hadoop 下文件上传到 input 目录下：</p>
<figure class="highlight plain"><table><tr><td class="gutter"><pre><div class="line">1</div><div class="line">2</div><div class="line">3</div></pre></td><td class="code"><pre><div class="line">bin/hdfs dfs -mkdir /user</div><div class="line">bin/hdfs dfs -mkdir /user/hadoop</div><div class="line">bin/hdfs dfs -put etc/hadoop input</div></pre></td></tr></table></figure>
<p><img src="http://upload-images.jianshu.io/upload_images/137261-bf6fc9e6ee002389.jpg?imageMogr2/auto-orient/strip%7CimageView2/2/w/1240" alt="向hdfs上传文件"></p>
<p>通过 web 方式可以看到文件已经上传成功：</p>
<p><img src="http://upload-images.jianshu.io/upload_images/137261-31e7db693f954734.jpg?imageMogr2/auto-orient/strip%7CimageView2/2/w/1240" alt="web方式查看上传文件"></p>
<p>同单机模式一样，运行 MapReduce 程序：</p>
<figure class="highlight plain"><table><tr><td class="gutter"><pre><div class="line">1</div></pre></td><td class="code"><pre><div class="line">bin/hadoop jar share/hadoop/mapreduce/hadoop-mapreduce-examples-2.6.5.jar grep input output &apos;dfs[a-z.]+&apos;</div></pre></td></tr></table></figure>
<blockquote>
<p>注意：运行前若 output 目录已存在则会报错，可执行 bin/hdfs dfs -rm -R /user/hadoop/output 删除该目录再重新运行</p>
</blockquote>
<p>程序运行结束后再执行 bin/hdfs dfs -cat output/* 查看上述程序的输出结果，效果如图：</p>
<p><img src="http://upload-images.jianshu.io/upload_images/137261-342e53f177650700.jpg?imageMogr2/auto-orient/strip%7CimageView2/2/w/1240" alt="程序输出结果"></p>
<p>使用中可以体会到 bin/hdfs dfs 的linux命令就是分布式文件系统的命令，类似于在linux本机上操作，只不过前面加了 bin/hdfs dfs，比如还有</p>
<figure class="highlight plain"><table><tr><td class="gutter"><pre><div class="line">1</div><div class="line">2</div><div class="line">3</div></pre></td><td class="code"><pre><div class="line">bin/hdfs dfs -ls /user/hadoop           # 查看 /user/hadoop 中的文件</div><div class="line">bin/hdfs dfs -rm -R /user/hadoop/input/*    # 删除 input 中的文件</div><div class="line">bin/hdfs dfs -rm -R /user/hadoop/output     # 删除 output 文件夹</div></pre></td></tr></table></figure>
<h4 id="停止-Hadoop"><a href="#停止-Hadoop" class="headerlink" title="停止 Hadoop"></a>停止 Hadoop</h4><p>执行如下命令即可停止 Hadoop：</p>
<figure class="highlight plain"><table><tr><td class="gutter"><pre><div class="line">1</div></pre></td><td class="code"><pre><div class="line">sbin/stop-dfs.sh</div></pre></td></tr></table></figure>
<blockquote>
<p>注意：下次再启动 Hadoop，无须再进行 hdfs 的初始化，只需要运行 <code>sbin/start-dfs.sh</code> 即可</p>
</blockquote>

      
    </div>
    
    
    

    

    
      <div>
        <div style="padding: 10px 0; margin: 20px auto; width: 90%; text-align: center;">
  <div>请我喝杯咖啡吧☕~</div>
  <button id="rewardButton" disable="enable" onclick="var qr = document.getElementById('QR'); if (qr.style.display === 'none') {qr.style.display='block';} else {qr.style.display='none'}">
    <span>打赏</span>
  </button>
  <div id="QR" style="display: none;">

    
      <div id="wechat" style="display: inline-block">
        <img id="wechat_qr" src="/images/wechatpay.jpg" alt="Fighter. 微信支付"/>
        <p>微信支付</p>
      </div>
    

    
      <div id="alipay" style="display: inline-block">
        <img id="alipay_qr" src="/images/alipay.jpg" alt="Fighter. 支付宝"/>
        <p>支付宝</p>
      </div>
    

    

  </div>
</div>

      </div>
    

    
      <div>
        <ul class="post-copyright">
  <li class="post-copyright-author">
    <strong>本文作者：</strong>
    Fighter.
  </li>
  <li class="post-copyright-link">
    <strong>本文链接：</strong>
    <a href="http://fighterhit.github.io/2017/11/12/other_notes/Hadoop-安装和使用/" title="Hadoop 安装和使用">http://fighterhit.github.io/2017/11/12/other_notes/Hadoop-安装和使用/</a>
  </li>
  <li class="post-copyright-license">
    <strong>版权声明： </strong>
    本博客所有文章除特别声明外，均采用 <a href="https://creativecommons.org/licenses/by-nc-sa/3.0/" rel="external nofollow" target="_blank">CC BY-NC-SA 3.0</a> 许可协议。转载请注明出处！
  </li>
</ul>

      </div>
    

    <footer class="post-footer">
      
        <div class="post-tags">
          
            <a href="/tags/Hadoop/" rel="tag"># Hadoop</a>
          
        </div>
      

      
      
      

      
        <div class="post-nav">
          <div class="post-nav-next post-nav-item">
            
              <a href="/2017/10/24/data_mining_notes/Python数据分析、挖掘常用工具/" rel="next" title="Python数据分析、挖掘常用工具">
                <i class="fa fa-chevron-left"></i> Python数据分析、挖掘常用工具
              </a>
            
          </div>

          <span class="post-nav-divider"></span>

          <div class="post-nav-prev post-nav-item">
            
              <a href="/2017/12/24/machine_learning_notes/从广义线性模型理解逻辑回归/" rel="prev" title="从广义线性模型(GLM)理解逻辑回归">
                从广义线性模型(GLM)理解逻辑回归 <i class="fa fa-chevron-right"></i>
              </a>
            
          </div>
        </div>
      

      
      
    </footer>
  </div>
  
  
  
  </article>



    <div class="post-spread">
      
        <!-- JiaThis Button BEGIN -->
<div class="jiathis_style">
<span class="jiathis_txt">分享到：</span>
<a class="jiathis_button_fav">收藏夹</a>
<a class="jiathis_button_tsina">新浪微博</a>
<a class="jiathis_button_qzone">QQ空间</a>
<a class="jiathis_button_weixin">微信</a>
<a class="jiathis_button_cqq">腾讯QQ</a>
<a class="jiathis_button_copy">复制网址</a>
<a class="jiathis_button_share">一键分享</a>

<a href="http://www.jiathis.com/share?uid=2140465" class="jiathis jiathis_txt jiathis_separator jtico jtico_jiathis" target="_blank">更多</a>
<a class="jiathis_counter_style"></a>
</div>
<script type="text/javascript" >
var jiathis_config={
  data_track_clickback:true,
  summary:"",
  shortUrl:false,
  hideMore:false
}
</script>
<script type="text/javascript" src="http://v3.jiathis.com/code/jia.js?uid=" charset="utf-8"></script>
<!-- JiaThis Button END -->
      
    </div>
  </div>


          </div>
          


          
  
    <div class="comments" id="comments">
      <div id="lv-container" data-id="city" data-uid="MTAyMC8zMjY0Ni85MjA3"></div>
    </div>
  


        </div>
        
          
  
  <div class="sidebar-toggle">
    <div class="sidebar-toggle-line-wrap">
      <span class="sidebar-toggle-line sidebar-toggle-line-first"></span>
      <span class="sidebar-toggle-line sidebar-toggle-line-middle"></span>
      <span class="sidebar-toggle-line sidebar-toggle-line-last"></span>
    </div>
  </div>

  <aside id="sidebar" class="sidebar">
    
    <div class="sidebar-inner">

      

      
        <ul class="sidebar-nav motion-element">
          <li class="sidebar-nav-toc sidebar-nav-active" data-target="post-toc-wrap">
            文章目录
          </li>
          <li class="sidebar-nav-overview" data-target="site-overview-wrap">
            站点概览
          </li>
        </ul>
      

      <section class="site-overview-wrap sidebar-panel">
        <div class="site-overview">
          <div class="site-author motion-element" itemprop="author" itemscope itemtype="http://schema.org/Person">
            
              <img class="site-author-image" itemprop="image"
                src="/images/avatar.jpg"
                alt="Fighter." />
            
              <p class="site-author-name" itemprop="name">Fighter.</p>
              <p class="site-description motion-element" itemprop="description">学习 | 分享 | 交流 | 进步</p>
          </div>

          <nav class="site-state motion-element">

            
              <div class="site-state-item site-state-posts">
              
                <a href="/archives/">
              
                  <span class="site-state-item-count">26</span>
                  <span class="site-state-item-name">日志</span>
                </a>
              </div>
            

            
              
              
              <div class="site-state-item site-state-categories">
                <a href="/categories/index.html">
                  <span class="site-state-item-count">6</span>
                  <span class="site-state-item-name">分类</span>
                </a>
              </div>
            

            
              
              
              <div class="site-state-item site-state-tags">
                <a href="/tags/index.html">
                  <span class="site-state-item-count">28</span>
                  <span class="site-state-item-name">标签</span>
                </a>
              </div>
            

          </nav>

          

          <div class="links-of-author motion-element">
            
              
                <span class="links-of-author-item">
                  <a href="https://github.com/fighterhit" target="_blank" title="GitHub">
                    
                      <i class="fa fa-fw fa-github"></i>GitHub</a>
                </span>
              
                <span class="links-of-author-item">
                  <a href="https://weibo.com/fighterhit" target="_blank" title="Weibo">
                    
                      <i class="fa fa-fw fa-weibo"></i>Weibo</a>
                </span>
              
            
          </div>

          
          

          
          

          

        </div>
      </section>

      
      <!--noindex-->
        <section class="post-toc-wrap motion-element sidebar-panel sidebar-panel-active">
          <div class="post-toc">

            
              
            

            
              <div class="post-toc-content"><ol class="nav"><li class="nav-item nav-level-2"><a class="nav-link" href="#环境"><span class="nav-number">1.</span> <span class="nav-text">环境</span></a></li><li class="nav-item nav-level-2"><a class="nav-link" href="#创建Hadoop用户"><span class="nav-number">2.</span> <span class="nav-text">创建Hadoop用户</span></a><ol class="nav-child"><li class="nav-item nav-level-4"><a class="nav-link" href="#创建用户"><span class="nav-number">2.0.1.</span> <span class="nav-text">创建用户</span></a></li><li class="nav-item nav-level-4"><a class="nav-link" href="#为新用户设置密码"><span class="nav-number">2.0.2.</span> <span class="nav-text">为新用户设置密码</span></a></li><li class="nav-item nav-level-4"><a class="nav-link" href="#为新用户添加权限"><span class="nav-number">2.0.3.</span> <span class="nav-text">为新用户添加权限</span></a></li></ol></li></ol></li><li class="nav-item nav-level-2"><a class="nav-link" href="#安装SSH-Server、配置SSH免密钥登录"><span class="nav-number">3.</span> <span class="nav-text">安装SSH Server、配置SSH免密钥登录</span></a><ol class="nav-child"><li class="nav-item nav-level-4"><a class="nav-link" href="#设置登录到本机"><span class="nav-number">3.0.1.</span> <span class="nav-text">设置登录到本机</span></a></li><li class="nav-item nav-level-4"><a class="nav-link" href="#配置免密钥登录"><span class="nav-number">3.0.2.</span> <span class="nav-text">配置免密钥登录</span></a></li></ol></li></ol></li><li class="nav-item nav-level-2"><a class="nav-link" href="#配置Java环境"><span class="nav-number">4.</span> <span class="nav-text">配置Java环境</span></a><ol class="nav-child"><li class="nav-item nav-level-4"><a class="nav-link" href="#下载JDK"><span class="nav-number">4.0.1.</span> <span class="nav-text">下载JDK</span></a></li><li class="nav-item nav-level-4"><a class="nav-link" href="#配置JDK环境变量"><span class="nav-number">4.0.2.</span> <span class="nav-text">配置JDK环境变量</span></a></li></ol></li></ol></li><li class="nav-item nav-level-2"><a class="nav-link" href="#安装、配置-Hadoop-2-6-5"><span class="nav-number">5.</span> <span class="nav-text">安装、配置 Hadoop 2.6.5</span></a><ol class="nav-child"><li class="nav-item nav-level-4"><a class="nav-link" href="#下载-Hadoop"><span class="nav-number">5.0.1.</span> <span class="nav-text">下载 Hadoop</span></a></li><li class="nav-item nav-level-4"><a class="nav-link" href="#Hadoop单机配置"><span class="nav-number">5.0.2.</span> <span class="nav-text">Hadoop单机配置</span></a></li><li class="nav-item nav-level-4"><a class="nav-link" href="#Hadoop伪分布式配置"><span class="nav-number">5.0.3.</span> <span class="nav-text">Hadoop伪分布式配置</span></a><ol class="nav-child"><li class="nav-item nav-level-5"><a class="nav-link" href="#修改配置文件-etc-hadoop-core-site-xml"><span class="nav-number">5.0.3.1.</span> <span class="nav-text">修改配置文件 etc/hadoop/core-site.xml</span></a></li><li class="nav-item nav-level-5"><a class="nav-link" href="#修改配置文件etc-hadoop-hdfs-site-xml"><span class="nav-number">5.0.3.2.</span> <span class="nav-text">修改配置文件etc/hadoop/hdfs-site.xml</span></a></li><li class="nav-item nav-level-5"><a class="nav-link" href="#Hadoop伪分布式实例-WordCount"><span class="nav-number">5.0.3.3.</span> <span class="nav-text">Hadoop伪分布式实例-WordCount</span></a></li></ol></li><li class="nav-item nav-level-4"><a class="nav-link" href="#停止-Hadoop"><span class="nav-number">5.0.4.</span> <span class="nav-text">停止 Hadoop</span></a></li></ol></li></ol></li></ol></div>
            

          </div>
        </section>
      <!--/noindex-->
      

      
      
    </div>
  </aside>


        
      </div>
    </main>

    <footer id="footer" class="footer">
      <div class="footer-inner">
        <div class="copyright">&copy; 2016 - <span itemprop="copyrightYear">2018</span>
  <span class="with-love">
    <i class="fa fa-home"></i>
  </span>
  <span class="author" itemprop="copyrightHolder">Fighter.</span>

  
</div>









        
<div class="busuanzi-count">
  <!--<script async src="https://dn-lbstatics.qbox.me/busuanzi/2.3/busuanzi.pure.mini.js"></script>-->
  <script async src="https://busuanzi.ibruce.info/busuanzi/2.3/busuanzi.pure.mini.js"></script>

  
    <span class="site-uv">
      <i class="fa fa-user"></i>
      <span class="busuanzi-value" id="busuanzi_value_site_uv"></span>
      
    </span>
  

  
    <span class="site-pv">
      <i class="fa fa-eye"></i>
      <span class="busuanzi-value" id="busuanzi_value_site_pv"></span>
      
    </span>
  

    <span class="site-pv">&nbsp;&nbsp;&nbsp;|&nbsp;&nbsp;&nbsp;
    Hosted by <a target="_blank" href="https://github.com/">GitHub Pages</a>
    </span>

</div>







        
      </div>
    </footer>

    
      <div class="back-to-top">
        <i class="fa fa-arrow-up"></i>
        
      </div>
    

  </div>

  

<script type="text/javascript">
  if (Object.prototype.toString.call(window.Promise) !== '[object Function]') {
    window.Promise = null;
  }
</script>









  












  
  <script type="text/javascript" src="/lib/jquery/index.js?v=2.1.3"></script>

  
  <script type="text/javascript" src="/lib/fastclick/lib/fastclick.min.js?v=1.0.6"></script>

  
  <script type="text/javascript" src="/lib/jquery_lazyload/jquery.lazyload.js?v=1.9.7"></script>

  
  <script type="text/javascript" src="/lib/velocity/velocity.min.js?v=1.2.1"></script>

  
  <script type="text/javascript" src="/lib/velocity/velocity.ui.min.js?v=1.2.1"></script>

  
  <script type="text/javascript" src="/lib/fancybox/source/jquery.fancybox.pack.js?v=2.1.5"></script>


  


  <script type="text/javascript" src="/js/src/utils.js?v=5.1.3"></script>

  <script type="text/javascript" src="/js/src/motion.js?v=5.1.3"></script>



  
  


  <script type="text/javascript" src="/js/src/affix.js?v=5.1.3"></script>

  <script type="text/javascript" src="/js/src/schemes/pisces.js?v=5.1.3"></script>



  
  <script type="text/javascript" src="/js/src/scrollspy.js?v=5.1.3"></script>
<script type="text/javascript" src="/js/src/post-details.js?v=5.1.3"></script>



  


  <script type="text/javascript" src="/js/src/bootstrap.js?v=5.1.3"></script>



  


  




	





  





  
    <script type="text/javascript">
      (function(d, s) {
        var j, e = d.getElementsByTagName(s)[0];
        if (typeof LivereTower === 'function') { return; }
        j = d.createElement(s);
        j.src = 'https://cdn-city.livere.com/js/embed.dist.js';
        j.async = true;
        e.parentNode.insertBefore(j, e);
      })(document, 'script');
    </script>
  








  





  

  
<script>
(function(){
    var bp = document.createElement('script');
    var curProtocol = window.location.protocol.split(':')[0];
    if (curProtocol === 'https') {
        bp.src = 'https://zz.bdstatic.com/linksubmit/push.js';        
    }
    else {
        bp.src = 'http://push.zhanzhang.baidu.com/push.js';
    }
    var s = document.getElementsByTagName("script")[0];
    s.parentNode.insertBefore(bp, s);
})();
</script><!-- hexo-inject:begin --><!-- hexo-inject:end -->


  

  
  


  

  

</body>
</html>
